# -*- coding: utf-8 -*-
import scrapy
from ncep.items import RadarItem

base_reflplus = 'https://mrms.ncep.noaa.gov/data/3DRefl/'
base_echotop = 'https://mrms.ncep.noaa.gov/data/2D/'

class RadarSpider(scrapy.Spider):
    name = 'radar'
    allowed_domains = ['mrms.ncep.noaa.gov']
    start_urls = ['http://mrms.ncep.noaa.gov/data']

    def parse(self, response):
        temp_reflplus = 'MergedReflectivityQC_'
        list_time = ['00.50/','00.75/','01.00/','01.25/','01.50/','01.75/','02.00/','02.25/','02.50/','02.75/','03.00/',
                    '03.50/','04.00/','04.50/','05.00/','05.50/','06.00/','06.50/','07.00/','07.50/','08.00/','08.50/',
                    '09.00/','10.00/','11.00/','12.00/','13.00/','14.00/','15.00/','16.00/','17.00/','18.00/','19.00/',]
        for str_time in list_time:
            url = base_reflplus + temp_reflplus + str_time
            # print('url=============', url)
            yield scrapy.Request(url, callback=self.detail_parse)

        list_echotop = ['EchoTop_18/', 'EchoTop_30/', 'EchoTop_50/', 'EchoTop_60/']
        for str_echotop in list_echotop:
            url = base_echotop + str_echotop
            yield scrapy.Request(url, callback=self.detail_parse)

    def detail_parse(self, response):
        base_url = response.url
        # print('base_url=============', base_url)
        hrefs = response.xpath('//tr/td[1]/a/text()').extract()

        # print('hrefs=============', hrefs)
        for href in hrefs:
            if href.endswith('.grib2.gz'):
                item = RadarItem()
                item['url'] = base_url + href
                # print('item=============', item)
                yield item
